{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "46dd3839",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-10-05T18:02:16.364530Z",
     "iopub.status.busy": "2024-10-05T18:02:16.363365Z",
     "iopub.status.idle": "2024-10-05T18:02:34.914942Z",
     "shell.execute_reply": "2024-10-05T18:02:34.913440Z"
    },
    "papermill": {
     "duration": 18.561218,
     "end_time": "2024-10-05T18:02:34.918172",
     "exception": false,
     "start_time": "2024-10-05T18:02:16.356954",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\u001b[31mERROR: pip's dependency resolver does not currently take into account all the packages that are installed. This behaviour is the source of the following dependency conflicts.\r\n",
      "bigframes 0.22.0 requires google-cloud-bigquery[bqstorage,pandas]>=3.10.0, but you have google-cloud-bigquery 2.34.4 which is incompatible.\r\n",
      "bigframes 0.22.0 requires google-cloud-storage>=2.0.0, but you have google-cloud-storage 1.44.0 which is incompatible.\r\n",
      "bigframes 0.22.0 requires pandas<2.1.4,>=1.5.0, but you have pandas 2.2.3 which is incompatible.\r\n",
      "cesium 0.12.3 requires numpy<3.0,>=2.0, but you have numpy 1.26.4 which is incompatible.\r\n",
      "dataproc-jupyter-plugin 0.1.79 requires pydantic~=1.10.0, but you have pydantic 2.9.2 which is incompatible.\u001b[0m\u001b[31m\r\n",
      "\u001b[0m"
     ]
    }
   ],
   "source": [
    "!pip install scikit-learn -U -q"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "b0f1de5e",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-10-05T18:02:34.929049Z",
     "iopub.status.busy": "2024-10-05T18:02:34.928589Z",
     "iopub.status.idle": "2024-10-05T18:02:37.639627Z",
     "shell.execute_reply": "2024-10-05T18:02:37.638532Z"
    },
    "papermill": {
     "duration": 2.719634,
     "end_time": "2024-10-05T18:02:37.642358",
     "exception": false,
     "start_time": "2024-10-05T18:02:34.922724",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "# Importing core libraries\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "from time import time\n",
    "import joblib\n",
    "\n",
    "# Suppressing warnings because of skopt verbosity\n",
    "import warnings\n",
    "warnings.filterwarnings(\"ignore\")\n",
    "\n",
    "# Classifier/Regressor\n",
    "from xgboost import XGBRegressor\n",
    "\n",
    "# Model selection\n",
    "from sklearn.model_selection import KFold\n",
    "\n",
    "# Metrics\n",
    "from sklearn.metrics import root_mean_squared_error\n",
    "\n",
    "# Skopt functions\n",
    "from skopt.space import Real, Integer\n",
    "from skopt.utils import use_named_args # decorator to convert a list of parameters to named arguments\n",
    "from skopt import forest_minimize, dummy_minimize\n",
    "\n",
    "# Data processing\n",
    "from sklearn.preprocessing import OrdinalEncoder"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "299941a7",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-10-05T18:02:37.652774Z",
     "iopub.status.busy": "2024-10-05T18:02:37.652124Z",
     "iopub.status.idle": "2024-10-05T18:02:43.536188Z",
     "shell.execute_reply": "2024-10-05T18:02:43.534947Z"
    },
    "papermill": {
     "duration": 5.892323,
     "end_time": "2024-10-05T18:02:43.539029",
     "exception": false,
     "start_time": "2024-10-05T18:02:37.646706",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "# Loading data \n",
    "X_train = pd.read_csv(\"../input/30-days-of-ml/train.csv\")\n",
    "X_test = pd.read_csv(\"../input/30-days-of-ml/test.csv\")\n",
    "\n",
    "# Preparing data as a tabular matrix\n",
    "y_train = X_train.target\n",
    "X_train = X_train.set_index('id').drop('target', axis='columns')\n",
    "X_test = X_test.set_index('id')\n",
    "\n",
    "# Pointing out categorical features\n",
    "categoricals = [item for item in X_train.columns if 'cat' in item]\n",
    "\n",
    "# Dealing with categorical data using OrdinalEncoder\n",
    "ordinal_encoder = OrdinalEncoder()\n",
    "X_train[categoricals] = ordinal_encoder.fit_transform(X_train[categoricals])\n",
    "X_test[categoricals] = ordinal_encoder.transform(X_test[categoricals])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "a2309291",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-10-05T18:02:43.551289Z",
     "iopub.status.busy": "2024-10-05T18:02:43.550819Z",
     "iopub.status.idle": "2024-10-05T18:02:43.556056Z",
     "shell.execute_reply": "2024-10-05T18:02:43.554652Z"
    },
    "papermill": {
     "duration": 0.015393,
     "end_time": "2024-10-05T18:02:43.558881",
     "exception": false,
     "start_time": "2024-10-05T18:02:43.543488",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "# Setting the scoring function\n",
    "scoring = root_mean_squared_error"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "8fb6ef61",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-10-05T18:02:43.570689Z",
     "iopub.status.busy": "2024-10-05T18:02:43.570258Z",
     "iopub.status.idle": "2024-10-05T18:02:43.575811Z",
     "shell.execute_reply": "2024-10-05T18:02:43.574681Z"
    },
    "papermill": {
     "duration": 0.014092,
     "end_time": "2024-10-05T18:02:43.578255",
     "exception": false,
     "start_time": "2024-10-05T18:02:43.564163",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "# Setting the cv strategy\n",
    "kf = KFold(n_splits=5, shuffle=True, random_state=0)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "da50d9ba",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-10-05T18:02:43.588378Z",
     "iopub.status.busy": "2024-10-05T18:02:43.587922Z",
     "iopub.status.idle": "2024-10-05T18:02:43.601782Z",
     "shell.execute_reply": "2024-10-05T18:02:43.600565Z"
    },
    "papermill": {
     "duration": 0.021787,
     "end_time": "2024-10-05T18:02:43.604264",
     "exception": false,
     "start_time": "2024-10-05T18:02:43.582477",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "# Setting the search space\n",
    "space = [Real(0.01, 1.0, 'uniform', name='learning_rate'),\n",
    "         Integer(1, 8, name='max_depth'),\n",
    "         Real(0.1, 1.0, 'uniform', name='subsample'),\n",
    "         Real(0.1, 1.0, 'uniform', name='colsample_bytree'),  # subsample ratio of columns by tree\n",
    "         Real(0, 100., 'uniform', name='reg_lambda'),      # L2 regularization\n",
    "         Real(0, 100., 'uniform', name='reg_alpha'),       # L1 regularization\n",
    "         Real(1, 30, 'uniform', name='min_child_weight'),     # minimum sum of instance weight (hessian)\n",
    "]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "cbc1e8eb",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-10-05T18:02:43.614361Z",
     "iopub.status.busy": "2024-10-05T18:02:43.613934Z",
     "iopub.status.idle": "2024-10-05T18:02:43.628422Z",
     "shell.execute_reply": "2024-10-05T18:02:43.627383Z"
    },
    "papermill": {
     "duration": 0.022206,
     "end_time": "2024-10-05T18:02:43.630651",
     "exception": false,
     "start_time": "2024-10-05T18:02:43.608445",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "counter = 0\n",
    "history = {i:list() for i in range(kf.get_n_splits())}\n",
    "used_time = 0\n",
    "\n",
    "from time import time\n",
    "\n",
    "def onstep(res):\n",
    "    global counter\n",
    "    x0 = res.x_iters   # List of input points\n",
    "    y0 = res.func_vals # Evaluation of input points\n",
    "    print('Last eval: ', x0[-1], \n",
    "          ' - Score ', y0[-1])\n",
    "    print('Current iter: ', counter, \n",
    "          ' - Best Score ', res.fun, \n",
    "          ' - Best Args: ', res.x)\n",
    "    joblib.dump((x0, y0), 'checkpoint.pkl') # Saving a checkpoint to disk\n",
    "    counter += 1\n",
    "    \n",
    "# The objective function to be minimized\n",
    "def make_objective(model, X, y, space, cv, scoring, validation=0.2):\n",
    "    # This decorator converts your objective function with named arguments into one that\n",
    "    # accepts a list as argument, while doing the conversion automatically.\n",
    "    @use_named_args(space) \n",
    "    def objective(**params):\n",
    "        model.set_params(**params)\n",
    "        print(\"\\nTesting: \", params)\n",
    "        validation_scores = list()\n",
    "        for k, (train_index, test_index) in enumerate(kf.split(X, y)):\n",
    "            val_index = list()\n",
    "            train_examples = len(train_index)\n",
    "            train_examples = int(train_examples * (1 - validation))\n",
    "            train_index, val_index = train_index[:train_examples], train_index[train_examples:]\n",
    "            \n",
    "            start_time = time()\n",
    "            model.fit(X.iloc[train_index,:], y[train_index],\n",
    "                      early_stopping_rounds=50,\n",
    "                      eval_set=[(X.iloc[val_index,:], y[val_index])], \n",
    "                      verbose=0\n",
    "                    )\n",
    "            end_time = time()\n",
    "            \n",
    "            rounds = model.best_iteration\n",
    "            \n",
    "            test_preds = model.predict(X.iloc[test_index,:])\n",
    "            test_score = scoring(y[test_index], test_preds)\n",
    "            print(f\"CV Fold {k+1} rmse:{test_score:0.5f} - {rounds} rounds - it took {end_time-start_time:0.0f} secs\")\n",
    "            validation_scores.append(test_score)\n",
    "            \n",
    "            if len(history[k]) >= 10:\n",
    "                threshold = np.percentile(history[k], q=25)\n",
    "                if test_score > threshold:\n",
    "                    print(f\"Early stopping for under-performing fold: threshold is {threshold:0.5f}\")\n",
    "                    return np.mean(validation_scores)\n",
    "                \n",
    "            history[k].append(test_score)\n",
    "        return np.mean(validation_scores)\n",
    "\n",
    "    return objective"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "c9eddb5c",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-10-05T18:02:43.640817Z",
     "iopub.status.busy": "2024-10-05T18:02:43.640388Z",
     "iopub.status.idle": "2024-10-05T18:02:43.645691Z",
     "shell.execute_reply": "2024-10-05T18:02:43.644393Z"
    },
    "papermill": {
     "duration": 0.013012,
     "end_time": "2024-10-05T18:02:43.647947",
     "exception": false,
     "start_time": "2024-10-05T18:02:43.634935",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "model = XGBRegressor(n_estimators=10_000, booster='gbtree', random_state=0)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "id": "61c999c6",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-10-05T18:02:43.657973Z",
     "iopub.status.busy": "2024-10-05T18:02:43.657540Z",
     "iopub.status.idle": "2024-10-05T18:02:43.662983Z",
     "shell.execute_reply": "2024-10-05T18:02:43.661876Z"
    },
    "papermill": {
     "duration": 0.013071,
     "end_time": "2024-10-05T18:02:43.665200",
     "exception": false,
     "start_time": "2024-10-05T18:02:43.652129",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "objective = make_objective(model,\n",
    "                           X_train, y_train,\n",
    "                           space=space,\n",
    "                           cv=kf,\n",
    "                           scoring=scoring)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "id": "5fc2a041",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-10-05T18:02:43.676702Z",
     "iopub.status.busy": "2024-10-05T18:02:43.675819Z",
     "iopub.status.idle": "2024-10-05T18:31:27.866856Z",
     "shell.execute_reply": "2024-10-05T18:31:27.865620Z"
    },
    "papermill": {
     "duration": 1724.200096,
     "end_time": "2024-10-05T18:31:27.869555",
     "exception": false,
     "start_time": "2024-10-05T18:02:43.669459",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "Testing:  {'learning_rate': 0.5969161720427683, 'max_depth': 1, 'subsample': 0.6424870384644795, 'colsample_bytree': 0.5903948646972073, 'reg_lambda': 42.36547993389048, 'reg_alpha': 64.58941130666562, 'min_child_weight': 13.690029126618084}\n",
      "CV Fold 1 rmse:0.72268 - 2028 rounds - it took 39 secs\n",
      "CV Fold 2 rmse:0.71930 - 2460 rounds - it took 50 secs\n",
      "CV Fold 3 rmse:0.72453 - 2159 rounds - it took 42 secs\n",
      "CV Fold 4 rmse:0.71893 - 1547 rounds - it took 31 secs\n",
      "CV Fold 5 rmse:0.71871 - 2106 rounds - it took 41 secs\n",
      "Last eval:  [0.5969161720427683, 1, 0.6424870384644795, 0.5903948646972073, 42.36547993389048, 64.58941130666562, 13.690029126618084]  - Score  0.7208291849569098\n",
      "Current iter:  0  - Best Score  0.7208291849569098  - Best Args:  [0.5969161720427683, 1, 0.6424870384644795, 0.5903948646972073, 42.36547993389048, 64.58941130666562, 13.690029126618084]\n",
      "\n",
      "Testing:  {'learning_rate': 0.8928552707742591, 'max_depth': 1, 'subsample': 0.3453906651221019, 'colsample_bytree': 0.529898605589215, 'reg_lambda': 81.21687287754933, 'reg_alpha': 47.99771723750574, 'min_child_weight': 12.390759086924064}\n",
      "CV Fold 1 rmse:0.72303 - 938 rounds - it took 20 secs\n",
      "CV Fold 2 rmse:0.72029 - 1129 rounds - it took 24 secs\n",
      "CV Fold 3 rmse:0.72480 - 1141 rounds - it took 22 secs\n",
      "CV Fold 4 rmse:0.71887 - 1014 rounds - it took 21 secs\n",
      "CV Fold 5 rmse:0.71986 - 841 rounds - it took 18 secs\n",
      "Last eval:  [0.8928552707742591, 1, 0.3453906651221019, 0.529898605589215, 81.21687287754933, 47.99771723750574, 12.390759086924064]  - Score  0.7213703131771615\n",
      "Current iter:  1  - Best Score  0.7208291849569098  - Best Args:  [0.5969161720427683, 1, 0.6424870384644795, 0.5903948646972073, 42.36547993389048, 64.58941130666562, 13.690029126618084]\n",
      "\n",
      "Testing:  {'learning_rate': 0.8377179759020038, 'max_depth': 1, 'subsample': 0.17841636973138664, 'colsample_bytree': 0.11819655769629316, 'reg_lambda': 83.26198455479381, 'reg_alpha': 77.81567509498505, 'min_child_weight': 26.230352299157758}\n",
      "CV Fold 1 rmse:0.72491 - 1678 rounds - it took 32 secs\n",
      "CV Fold 2 rmse:0.72384 - 995 rounds - it took 19 secs\n",
      "CV Fold 3 rmse:0.73013 - 625 rounds - it took 14 secs\n",
      "CV Fold 4 rmse:0.72241 - 920 rounds - it took 18 secs\n",
      "CV Fold 5 rmse:0.72267 - 966 rounds - it took 20 secs\n",
      "Last eval:  [0.8377179759020038, 1, 0.17841636973138664, 0.11819655769629316, 83.26198455479381, 77.81567509498505, 26.230352299157758]  - Score  0.7247898017187254\n",
      "Current iter:  2  - Best Score  0.7208291849569098  - Best Args:  [0.5969161720427683, 1, 0.6424870384644795, 0.5903948646972073, 42.36547993389048, 64.58941130666562, 13.690029126618084]\n",
      "\n",
      "Testing:  {'learning_rate': 0.9788321588104365, 'max_depth': 1, 'subsample': 0.8208196767816799, 'colsample_bytree': 0.5684297315960845, 'reg_lambda': 67.88795301189604, 'reg_alpha': 72.06326547259168, 'min_child_weight': 17.87857397017811}\n",
      "CV Fold 1 rmse:0.72162 - 2243 rounds - it took 43 secs\n",
      "CV Fold 2 rmse:0.71897 - 2222 rounds - it took 42 secs\n",
      "CV Fold 3 rmse:0.72420 - 1858 rounds - it took 37 secs\n",
      "CV Fold 4 rmse:0.71925 - 1033 rounds - it took 19 secs\n",
      "CV Fold 5 rmse:0.71854 - 1900 rounds - it took 37 secs\n",
      "Last eval:  [0.9788321588104365, 1, 0.8208196767816799, 0.5684297315960845, 67.88795301189604, 72.06326547259168, 17.87857397017811]  - Score  0.7205167449020294\n",
      "Current iter:  3  - Best Score  0.7205167449020294  - Best Args:  [0.9788321588104365, 1, 0.8208196767816799, 0.5684297315960845, 67.88795301189604, 72.06326547259168, 17.87857397017811]\n",
      "\n",
      "Testing:  {'learning_rate': 0.5419994971545207, 'max_depth': 1, 'subsample': 0.5696634895750646, 'colsample_bytree': 0.4731957459914713, 'reg_lambda': 26.4555612104627, 'reg_alpha': 77.42336894342168, 'min_child_weight': 14.22835963427991}\n",
      "CV Fold 1 rmse:0.72231 - 2718 rounds - it took 52 secs\n",
      "CV Fold 2 rmse:0.72005 - 2204 rounds - it took 42 secs\n",
      "CV Fold 3 rmse:0.72463 - 2471 rounds - it took 49 secs\n",
      "CV Fold 4 rmse:0.71858 - 2064 rounds - it took 40 secs\n",
      "CV Fold 5 rmse:0.71928 - 2134 rounds - it took 41 secs\n",
      "Last eval:  [0.5419994971545207, 1, 0.5696634895750646, 0.4731957459914713, 26.4555612104627, 77.42336894342168, 14.22835963427991]  - Score  0.7209708343067436\n",
      "Current iter:  4  - Best Score  0.7205167449020294  - Best Args:  [0.9788321588104365, 1, 0.8208196767816799, 0.5684297315960845, 67.88795301189604, 72.06326547259168, 17.87857397017811]\n",
      "\n",
      "Testing:  {'learning_rate': 0.5727496093799621, 'max_depth': 1, 'subsample': 0.3917269070138927, 'colsample_bytree': 0.2347073804653149, 'reg_lambda': 22.232138825158767, 'reg_alpha': 38.6488981125862, 'min_child_weight': 27.17535579035274}\n",
      "CV Fold 1 rmse:0.72439 - 1072 rounds - it took 22 secs\n",
      "CV Fold 2 rmse:0.72098 - 1330 rounds - it took 26 secs\n",
      "CV Fold 3 rmse:0.72617 - 1241 rounds - it took 25 secs\n",
      "CV Fold 4 rmse:0.71947 - 1243 rounds - it took 23 secs\n",
      "CV Fold 5 rmse:0.72019 - 1205 rounds - it took 25 secs\n",
      "Last eval:  [0.5727496093799621, 1, 0.3917269070138927, 0.2347073804653149, 22.232138825158767, 38.6488981125862, 27.17535579035274]  - Score  0.7222407304753196\n",
      "Current iter:  5  - Best Score  0.7205167449020294  - Best Args:  [0.9788321588104365, 1, 0.8208196767816799, 0.5684297315960845, 67.88795301189604, 72.06326547259168, 17.87857397017811]\n",
      "\n",
      "Testing:  {'learning_rate': 0.45545049001211535, 'max_depth': 8, 'subsample': 0.4933287584194074, 'colsample_bytree': 0.7278680763345384, 'reg_lambda': 6.022547162926984, 'reg_alpha': 66.67667154456677, 'min_child_weight': 20.448498218926627}\n",
      "CV Fold 1 rmse:0.72837 - 43 rounds - it took 4 secs\n",
      "CV Fold 2 rmse:0.72560 - 54 rounds - it took 4 secs\n",
      "CV Fold 3 rmse:0.73053 - 59 rounds - it took 6 secs\n",
      "CV Fold 4 rmse:0.72437 - 52 rounds - it took 4 secs\n",
      "CV Fold 5 rmse:0.72529 - 53 rounds - it took 4 secs\n",
      "Last eval:  [0.45545049001211535, 8, 0.4933287584194074, 0.7278680763345384, 6.022547162926984, 66.67667154456677, 20.448498218926627]  - Score  0.7268325952952612\n",
      "Current iter:  6  - Best Score  0.7205167449020294  - Best Args:  [0.9788321588104365, 1, 0.8208196767816799, 0.5684297315960845, 67.88795301189604, 72.06326547259168, 17.87857397017811]\n",
      "\n",
      "Testing:  {'learning_rate': 0.21827873546310253, 'max_depth': 4, 'subsample': 0.7756175270966107, 'colsample_bytree': 0.6470476018439211, 'reg_lambda': 32.504722900835255, 'reg_alpha': 3.842542647273473, 'min_child_weight': 19.393947680762718}\n",
      "CV Fold 1 rmse:0.72341 - 336 rounds - it took 9 secs\n",
      "CV Fold 2 rmse:0.72076 - 291 rounds - it took 8 secs\n",
      "CV Fold 3 rmse:0.72531 - 245 rounds - it took 9 secs\n",
      "CV Fold 4 rmse:0.71933 - 379 rounds - it took 10 secs\n",
      "CV Fold 5 rmse:0.71943 - 363 rounds - it took 10 secs\n",
      "Last eval:  [0.21827873546310253, 4, 0.7756175270966107, 0.6470476018439211, 32.504722900835255, 3.842542647273473, 19.393947680762718]  - Score  0.7216470880184641\n",
      "Current iter:  7  - Best Score  0.7205167449020294  - Best Args:  [0.9788321588104365, 1, 0.8208196767816799, 0.5684297315960845, 67.88795301189604, 72.06326547259168, 17.87857397017811]\n",
      "\n",
      "Testing:  {'learning_rate': 0.9593597759382753, 'max_depth': 8, 'subsample': 0.28798908048535127, 'colsample_bytree': 0.24517856609649666, 'reg_lambda': 65.31083254653986, 'reg_alpha': 25.329160253978216, 'min_child_weight': 14.523012412832884}\n",
      "CV Fold 1 rmse:0.73646 - 18 rounds - it took 3 secs\n",
      "CV Fold 2 rmse:0.73374 - 8 rounds - it took 4 secs\n",
      "CV Fold 3 rmse:0.73848 - 17 rounds - it took 3 secs\n",
      "CV Fold 4 rmse:0.73010 - 8 rounds - it took 2 secs\n",
      "CV Fold 5 rmse:0.73205 - 15 rounds - it took 3 secs\n",
      "Last eval:  [0.9593597759382753, 8, 0.28798908048535127, 0.24517856609649666, 65.31083254653986, 25.329160253978216, 14.523012412832884]  - Score  0.7341658098156769\n",
      "Current iter:  8  - Best Score  0.7205167449020294  - Best Args:  [0.9788321588104365, 1, 0.8208196767816799, 0.5684297315960845, 67.88795301189604, 72.06326547259168, 17.87857397017811]\n",
      "\n",
      "Testing:  {'learning_rate': 0.25198133608158674, 'max_depth': 8, 'subsample': 0.4042068533550026, 'colsample_bytree': 0.7072770900331187, 'reg_lambda': 31.720174206929613, 'reg_alpha': 77.83454820259091, 'min_child_weight': 28.537560550071525}\n",
      "CV Fold 1 rmse:0.72634 - 159 rounds - it took 8 secs\n",
      "CV Fold 2 rmse:0.72350 - 130 rounds - it took 7 secs\n",
      "CV Fold 3 rmse:0.72870 - 151 rounds - it took 7 secs\n",
      "CV Fold 4 rmse:0.72256 - 148 rounds - it took 9 secs\n",
      "CV Fold 5 rmse:0.72299 - 213 rounds - it took 9 secs\n",
      "Last eval:  [0.25198133608158674, 8, 0.4042068533550026, 0.7072770900331187, 31.720174206929613, 77.83454820259091, 28.537560550071525]  - Score  0.724818463364308\n",
      "Current iter:  9  - Best Score  0.7205167449020294  - Best Args:  [0.9788321588104365, 1, 0.8208196767816799, 0.5684297315960845, 67.88795301189604, 72.06326547259168, 17.87857397017811]\n",
      "\n",
      "Testing:  {'learning_rate': 0.6659015982805438, 'max_depth': 7, 'subsample': 0.18739114821375516, 'colsample_bytree': 0.8541504167489236, 'reg_lambda': 9.609840789396308, 'reg_alpha': 97.6459465013396, 'min_child_weight': 14.590884847783347}\n",
      "CV Fold 1 rmse:0.73294 - 51 rounds - it took 4 secs\n",
      "Early stopping for under-performing fold: threshold is 0.72277\n",
      "Last eval:  [0.6659015982805438, 7, 0.18739114821375516, 0.8541504167489236, 9.609840789396308, 97.6459465013396, 14.590884847783347]  - Score  0.732937404093717\n",
      "Current iter:  10  - Best Score  0.7205167449020294  - Best Args:  [0.9788321588104365, 1, 0.8208196767816799, 0.5684297315960845, 67.88795301189604, 72.06326547259168, 17.87857397017811]\n",
      "\n",
      "Testing:  {'learning_rate': 0.9769934773084339, 'max_depth': 8, 'subsample': 0.5060432930688353, 'colsample_bytree': 0.11798889886788287, 'reg_lambda': 44.171092124884545, 'reg_alpha': 97.95867288127286, 'min_child_weight': 11.423889455110325}\n",
      "CV Fold 1 rmse:0.72351 - 224 rounds - it took 7 secs\n",
      "Early stopping for under-performing fold: threshold is 0.72277\n",
      "Last eval:  [0.9769934773084339, 8, 0.5060432930688353, 0.11798889886788287, 44.171092124884545, 97.95867288127286, 11.423889455110325]  - Score  0.7235085299893486\n",
      "Current iter:  11  - Best Score  0.7205167449020294  - Best Args:  [0.9788321588104365, 1, 0.8208196767816799, 0.5684297315960845, 67.88795301189604, 72.06326547259168, 17.87857397017811]\n",
      "\n",
      "Testing:  {'learning_rate': 0.48608459552780126, 'max_depth': 2, 'subsample': 0.3861848614545784, 'colsample_bytree': 0.472836695063203, 'reg_lambda': 6.414749634878437, 'reg_alpha': 69.24721193700199, 'min_child_weight': 17.431442171990682}\n",
      "CV Fold 1 rmse:0.72313 - 718 rounds - it took 19 secs\n",
      "Early stopping for under-performing fold: threshold is 0.72277\n",
      "Last eval:  [0.48608459552780126, 2, 0.3861848614545784, 0.472836695063203, 6.414749634878437, 69.24721193700199, 17.431442171990682]  - Score  0.7231260832366845\n",
      "Current iter:  12  - Best Score  0.7205167449020294  - Best Args:  [0.9788321588104365, 1, 0.8208196767816799, 0.5684297315960845, 67.88795301189604, 72.06326547259168, 17.87857397017811]\n",
      "\n",
      "Testing:  {'learning_rate': 0.272735596030051, 'max_depth': 4, 'subsample': 0.9250506586175148, 'colsample_bytree': 0.9290418492134799, 'reg_lambda': 8.31124926306024, 'reg_alpha': 27.77185612810325, 'min_child_weight': 1.2713444408394459}\n",
      "CV Fold 1 rmse:0.72287 - 299 rounds - it took 9 secs\n",
      "Early stopping for under-performing fold: threshold is 0.72277\n",
      "Last eval:  [0.272735596030051, 4, 0.9250506586175148, 0.9290418492134799, 8.31124926306024, 27.77185612810325, 1.2713444408394459]  - Score  0.7228671579720225\n",
      "Current iter:  13  - Best Score  0.7205167449020294  - Best Args:  [0.9788321588104365, 1, 0.8208196767816799, 0.5684297315960845, 67.88795301189604, 72.06326547259168, 17.87857397017811]\n",
      "\n",
      "Testing:  {'learning_rate': 0.8439186588710538, 'max_depth': 1, 'subsample': 0.7446944837067091, 'colsample_bytree': 0.360465483652481, 'reg_lambda': 18.319136200711686, 'reg_alpha': 58.651293481008324, 'min_child_weight': 1.5831188394373132}\n",
      "CV Fold 1 rmse:0.72190 - 2140 rounds - it took 42 secs\n",
      "CV Fold 2 rmse:0.71940 - 1988 rounds - it took 38 secs\n",
      "CV Fold 3 rmse:0.72507 - 1416 rounds - it took 28 secs\n",
      "Early stopping for under-performing fold: threshold is 0.72467\n",
      "Last eval:  [0.8439186588710538, 1, 0.7446944837067091, 0.360465483652481, 18.319136200711686, 58.651293481008324, 1.5831188394373132]  - Score  0.7221251916242761\n",
      "Current iter:  14  - Best Score  0.7205167449020294  - Best Args:  [0.9788321588104365, 1, 0.8208196767816799, 0.5684297315960845, 67.88795301189604, 72.06326547259168, 17.87857397017811]\n",
      "\n",
      "Testing:  {'learning_rate': 0.8306506289251896, 'max_depth': 7, 'subsample': 0.23179758645620105, 'colsample_bytree': 0.6126565653309761, 'reg_lambda': 70.37372792899164, 'reg_alpha': 28.84764370485287, 'min_child_weight': 13.565353797479279}\n",
      "CV Fold 1 rmse:0.73721 - 9 rounds - it took 3 secs\n",
      "Early stopping for under-performing fold: threshold is 0.72250\n",
      "Last eval:  [0.8306506289251896, 7, 0.23179758645620105, 0.6126565653309761, 70.37372792899164, 28.84764370485287, 13.565353797479279]  - Score  0.7372136996320826\n",
      "Current iter:  15  - Best Score  0.7205167449020294  - Best Args:  [0.9788321588104365, 1, 0.8208196767816799, 0.5684297315960845, 67.88795301189604, 72.06326547259168, 17.87857397017811]\n",
      "\n",
      "Testing:  {'learning_rate': 0.7585456269263906, 'max_depth': 8, 'subsample': 0.6328377381446552, 'colsample_bytree': 0.6150267152117861, 'reg_lambda': 22.308163264061836, 'reg_alpha': 95.27490115169851, 'min_child_weight': 13.966635979911194}\n",
      "CV Fold 1 rmse:0.72939 - 27 rounds - it took 3 secs\n",
      "Early stopping for under-performing fold: threshold is 0.72250\n",
      "Last eval:  [0.7585456269263906, 8, 0.6328377381446552, 0.6150267152117861, 22.308163264061836, 95.27490115169851, 13.966635979911194]  - Score  0.7293900226994231\n",
      "Current iter:  16  - Best Score  0.7205167449020294  - Best Args:  [0.9788321588104365, 1, 0.8208196767816799, 0.5684297315960845, 67.88795301189604, 72.06326547259168, 17.87857397017811]\n",
      "\n",
      "Testing:  {'learning_rate': 0.8479445857464166, 'max_depth': 6, 'subsample': 0.204581718651155, 'colsample_bytree': 0.7903213332028541, 'reg_lambda': 41.18201389648448, 'reg_alpha': 67.5439081200236, 'min_child_weight': 8.244091993994365}\n",
      "CV Fold 1 rmse:0.73393 - 23 rounds - it took 3 secs\n",
      "Early stopping for under-performing fold: threshold is 0.72250\n",
      "Last eval:  [0.8479445857464166, 6, 0.204581718651155, 0.7903213332028541, 41.18201389648448, 67.5439081200236, 8.244091993994365]  - Score  0.7339335117770172\n",
      "Current iter:  17  - Best Score  0.7205167449020294  - Best Args:  [0.9788321588104365, 1, 0.8208196767816799, 0.5684297315960845, 67.88795301189604, 72.06326547259168, 17.87857397017811]\n",
      "\n",
      "Testing:  {'learning_rate': 0.3200861486264915, 'max_depth': 7, 'subsample': 0.7232784310699893, 'colsample_bytree': 0.7527288518376766, 'reg_lambda': 50.132438192670236, 'reg_alpha': 95.6083634723224, 'min_child_weight': 19.675715777659487}\n",
      "CV Fold 1 rmse:0.72506 - 162 rounds - it took 7 secs\n",
      "Early stopping for under-performing fold: threshold is 0.72250\n",
      "Last eval:  [0.3200861486264915, 7, 0.7232784310699893, 0.7527288518376766, 50.132438192670236, 95.6083634723224, 19.675715777659487]  - Score  0.7250603156497097\n",
      "Current iter:  18  - Best Score  0.7205167449020294  - Best Args:  [0.9788321588104365, 1, 0.8208196767816799, 0.5684297315960845, 67.88795301189604, 72.06326547259168, 17.87857397017811]\n",
      "\n",
      "Testing:  {'learning_rate': 0.42961649807259794, 'max_depth': 4, 'subsample': 0.5273807638680479, 'colsample_bytree': 0.5231189703301887, 'reg_lambda': 71.60745312286433, 'reg_alpha': 28.799100436328054, 'min_child_weight': 12.12040452601349}\n",
      "CV Fold 1 rmse:0.72505 - 174 rounds - it took 6 secs\n",
      "Early stopping for under-performing fold: threshold is 0.72250\n",
      "Last eval:  [0.42961649807259794, 4, 0.5273807638680479, 0.5231189703301887, 71.60745312286433, 28.799100436328054, 12.12040452601349]  - Score  0.7250538028397069\n",
      "Current iter:  19  - Best Score  0.7205167449020294  - Best Args:  [0.9788321588104365, 1, 0.8208196767816799, 0.5684297315960845, 67.88795301189604, 72.06326547259168, 17.87857397017811]\n",
      "\n",
      "Testing:  {'learning_rate': 0.7516781387811975, 'max_depth': 1, 'subsample': 0.22192665780020523, 'colsample_bytree': 0.3684540933604278, 'reg_lambda': 56.996491070126496, 'reg_alpha': 59.08727612481733, 'min_child_weight': 17.655432216637788}\n",
      "CV Fold 1 rmse:0.72457 - 830 rounds - it took 18 secs\n",
      "Early stopping for under-performing fold: threshold is 0.72250\n",
      "Last eval:  [0.7516781387811975, 1, 0.22192665780020523, 0.3684540933604278, 56.996491070126496, 59.08727612481733, 17.655432216637788]  - Score  0.7245688358502311\n",
      "Current iter:  20  - Best Score  0.7205167449020294  - Best Args:  [0.9788321588104365, 1, 0.8208196767816799, 0.5684297315960845, 67.88795301189604, 72.06326547259168, 17.87857397017811]\n",
      "\n",
      "Testing:  {'learning_rate': 0.6566688116585624, 'max_depth': 6, 'subsample': 0.38889751723614785, 'colsample_bytree': 0.2991364166654045, 'reg_lambda': 14.126390492589671, 'reg_alpha': 9.72599270631739, 'min_child_weight': 29.537224999560248}\n",
      "CV Fold 1 rmse:0.73273 - 23 rounds - it took 3 secs\n",
      "Early stopping for under-performing fold: threshold is 0.72250\n",
      "Last eval:  [0.6566688116585624, 6, 0.38889751723614785, 0.2991364166654045, 14.126390492589671, 9.72599270631739, 29.537224999560248]  - Score  0.7327317799297081\n",
      "Current iter:  21  - Best Score  0.7205167449020294  - Best Args:  [0.9788321588104365, 1, 0.8208196767816799, 0.5684297315960845, 67.88795301189604, 72.06326547259168, 17.87857397017811]\n",
      "\n",
      "Testing:  {'learning_rate': 0.26773751831153064, 'max_depth': 6, 'subsample': 0.7334997251863298, 'colsample_bytree': 0.19020419858107102, 'reg_lambda': 91.94826137446736, 'reg_alpha': 71.42412995491115, 'min_child_weight': 29.966563190468133}\n",
      "CV Fold 1 rmse:0.72218 - 338 rounds - it took 12 secs\n",
      "CV Fold 2 rmse:0.71953 - 532 rounds - it took 18 secs\n",
      "CV Fold 3 rmse:0.72388 - 373 rounds - it took 12 secs\n",
      "CV Fold 4 rmse:0.71761 - 408 rounds - it took 15 secs\n",
      "CV Fold 5 rmse:0.71879 - 366 rounds - it took 12 secs\n",
      "Last eval:  [0.26773751831153064, 6, 0.7334997251863298, 0.19020419858107102, 91.94826137446736, 71.42412995491115, 29.966563190468133]  - Score  0.7204004344008961\n",
      "Current iter:  22  - Best Score  0.7204004344008961  - Best Args:  [0.26773751831153064, 6, 0.7334997251863298, 0.19020419858107102, 91.94826137446736, 71.42412995491115, 29.966563190468133]\n",
      "\n",
      "Testing:  {'learning_rate': 0.15795382161141383, 'max_depth': 8, 'subsample': 0.13380344544532627, 'colsample_bytree': 0.5574839037210019, 'reg_lambda': 16.684751304849968, 'reg_alpha': 77.90510196090358, 'min_child_weight': 26.08306557247229}\n",
      "CV Fold 1 rmse:0.72723 - 461 rounds - it took 17 secs\n",
      "Early stopping for under-performing fold: threshold is 0.72228\n",
      "Last eval:  [0.15795382161141383, 8, 0.13380344544532627, 0.5574839037210019, 16.684751304849968, 77.90510196090358, 26.08306557247229]  - Score  0.7272254961771806\n",
      "Current iter:  23  - Best Score  0.7204004344008961  - Best Args:  [0.26773751831153064, 6, 0.7334997251863298, 0.19020419858107102, 91.94826137446736, 71.42412995491115, 29.966563190468133]\n",
      "\n",
      "Testing:  {'learning_rate': 0.41728275651790003, 'max_depth': 4, 'subsample': 0.4664649675033997, 'colsample_bytree': 0.16225029590962425, 'reg_lambda': 69.74287731445638, 'reg_alpha': 45.354268267806894, 'min_child_weight': 21.939612384640093}\n",
      "CV Fold 1 rmse:0.72274 - 297 rounds - it took 9 secs\n",
      "Early stopping for under-performing fold: threshold is 0.72228\n",
      "Last eval:  [0.41728275651790003, 4, 0.4664649675033997, 0.16225029590962425, 69.74287731445638, 45.354268267806894, 21.939612384640093]  - Score  0.7227435448023911\n",
      "Current iter:  24  - Best Score  0.7204004344008961  - Best Args:  [0.26773751831153064, 6, 0.7334997251863298, 0.19020419858107102, 91.94826137446736, 71.42412995491115, 29.966563190468133]\n",
      "\n",
      "Testing:  {'learning_rate': 0.867718502669343, 'max_depth': 8, 'subsample': 0.11496517636048054, 'colsample_bytree': 0.30766810186611615, 'reg_lambda': 76.49116989968573, 'reg_alpha': 94.41235194354547, 'min_child_weight': 22.749978214332913}\n",
      "CV Fold 1 rmse:0.73315 - 58 rounds - it took 3 secs\n",
      "Early stopping for under-performing fold: threshold is 0.72228\n",
      "Last eval:  [0.867718502669343, 8, 0.11496517636048054, 0.30766810186611615, 76.49116989968573, 94.41235194354547, 22.749978214332913]  - Score  0.7331483425002626\n",
      "Current iter:  25  - Best Score  0.7204004344008961  - Best Args:  [0.26773751831153064, 6, 0.7334997251863298, 0.19020419858107102, 91.94826137446736, 71.42412995491115, 29.966563190468133]\n",
      "\n",
      "Testing:  {'learning_rate': 0.34600978125502346, 'max_depth': 1, 'subsample': 0.14890418950532827, 'colsample_bytree': 0.2799968724067601, 'reg_lambda': 1.8521794460613974, 'reg_alpha': 79.36977033574207, 'min_child_weight': 7.4938159537510245}\n",
      "CV Fold 1 rmse:0.72783 - 1096 rounds - it took 22 secs\n",
      "Early stopping for under-performing fold: threshold is 0.72228\n",
      "Last eval:  [0.34600978125502346, 1, 0.14890418950532827, 0.2799968724067601, 1.8521794460613974, 79.36977033574207, 7.4938159537510245]  - Score  0.7278342134553439\n",
      "Current iter:  26  - Best Score  0.7204004344008961  - Best Args:  [0.26773751831153064, 6, 0.7334997251863298, 0.19020419858107102, 91.94826137446736, 71.42412995491115, 29.966563190468133]\n",
      "\n",
      "Testing:  {'learning_rate': 0.3518981638899337, 'max_depth': 1, 'subsample': 0.6474275792686117, 'colsample_bytree': 0.6369898656655478, 'reg_lambda': 78.36442453018886, 'reg_alpha': 50.00262977293015, 'min_child_weight': 2.4607316459687705}\n",
      "CV Fold 1 rmse:0.72228 - 3653 rounds - it took 71 secs\n",
      "CV Fold 2 rmse:0.71900 - 4177 rounds - it took 77 secs\n",
      "CV Fold 3 rmse:0.72464 - 3373 rounds - it took 64 secs\n",
      "Early stopping for under-performing fold: threshold is 0.72458\n",
      "Last eval:  [0.3518981638899337, 1, 0.6474275792686117, 0.6369898656655478, 78.36442453018886, 50.00262977293015, 2.4607316459687705]  - Score  0.7219720534696585\n",
      "Current iter:  27  - Best Score  0.7204004344008961  - Best Args:  [0.26773751831153064, 6, 0.7334997251863298, 0.19020419858107102, 91.94826137446736, 71.42412995491115, 29.966563190468133]\n",
      "\n",
      "Testing:  {'learning_rate': 0.7021070939713261, 'max_depth': 1, 'subsample': 0.9407925981323145, 'colsample_bytree': 0.6525693603693065, 'reg_lambda': 53.56328030249584, 'reg_alpha': 58.990997635457106, 'min_child_weight': 22.173538855986322}\n",
      "CV Fold 1 rmse:0.72172 - 3682 rounds - it took 71 secs\n",
      "CV Fold 2 rmse:0.71967 - 2777 rounds - it took 52 secs\n",
      "Early stopping for under-performing fold: threshold is 0.71940\n",
      "Last eval:  [0.7021070939713261, 1, 0.9407925981323145, 0.6525693603693065, 53.56328030249584, 58.990997635457106, 22.173538855986322]  - Score  0.720698906524724\n",
      "Current iter:  28  - Best Score  0.7204004344008961  - Best Args:  [0.26773751831153064, 6, 0.7334997251863298, 0.19020419858107102, 91.94826137446736, 71.42412995491115, 29.966563190468133]\n",
      "\n",
      "Testing:  {'learning_rate': 0.3188255455248059, 'max_depth': 6, 'subsample': 0.29115144900127776, 'colsample_bytree': 0.3001989019481369, 'reg_lambda': 21.874937373677188, 'reg_alpha': 56.95735345747381, 'min_child_weight': 14.111162015969347}\n",
      "CV Fold 1 rmse:0.72542 - 167 rounds - it took 7 secs\n",
      "Early stopping for under-performing fold: threshold is 0.72220\n",
      "Last eval:  [0.3188255455248059, 6, 0.29115144900127776, 0.3001989019481369, 21.874937373677188, 56.95735345747381, 14.111162015969347]  - Score  0.7254204239839552\n",
      "Current iter:  29  - Best Score  0.7204004344008961  - Best Args:  [0.26773751831153064, 6, 0.7334997251863298, 0.19020419858107102, 91.94826137446736, 71.42412995491115, 29.966563190468133]\n"
     ]
    }
   ],
   "source": [
    "gp_round = dummy_minimize(func=objective,\n",
    "                          dimensions=space,\n",
    "                          n_calls=30,\n",
    "                          callback=[onstep],\n",
    "                          random_state=0)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "id": "37a04de4",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-10-05T18:31:27.891458Z",
     "iopub.status.busy": "2024-10-05T18:31:27.891061Z",
     "iopub.status.idle": "2024-10-05T18:31:27.899896Z",
     "shell.execute_reply": "2024-10-05T18:31:27.898666Z"
    },
    "papermill": {
     "duration": 0.023148,
     "end_time": "2024-10-05T18:31:27.902428",
     "exception": false,
     "start_time": "2024-10-05T18:31:27.879280",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "30\n"
     ]
    }
   ],
   "source": [
    "x0, y0 = joblib.load('checkpoint.pkl')\n",
    "print(len(x0))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "id": "35090e90",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-10-05T18:31:27.923200Z",
     "iopub.status.busy": "2024-10-05T18:31:27.922801Z",
     "iopub.status.idle": "2024-10-05T20:14:59.213601Z",
     "shell.execute_reply": "2024-10-05T20:14:59.212166Z"
    },
    "papermill": {
     "duration": 6211.30419,
     "end_time": "2024-10-05T20:14:59.216193",
     "exception": false,
     "start_time": "2024-10-05T18:31:27.912003",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Last eval:  [0.3188255455248059, 6, 0.29115144900127776, 0.3001989019481369, 21.874937373677188, 56.95735345747381, 14.111162015969347]  - Score  0.7254204239839552\n",
      "Current iter:  30  - Best Score  0.7204004344008961  - Best Args:  [0.26773751831153064, 6, 0.7334997251863298, 0.19020419858107102, 91.94826137446736, 71.42412995491115, 29.966563190468133]\n",
      "\n",
      "Testing:  {'learning_rate': 0.01, 'max_depth': 1, 'subsample': 1.0, 'colsample_bytree': 0.36533871151300934, 'reg_lambda': 49.53477478656924, 'reg_alpha': 81.28334197165063, 'min_child_weight': 30.0}\n",
      "CV Fold 1 rmse:0.73200 - 9999 rounds - it took 177 secs\n",
      "Early stopping for under-performing fold: threshold is 0.72220\n",
      "Last eval:  [0.01, 1, 1.0, 0.36533871151300934, 49.53477478656924, 81.28334197165063, 30.0]  - Score  0.7319956200383687\n",
      "Current iter:  31  - Best Score  0.7204004344008961  - Best Args:  [0.26773751831153064, 6, 0.7334997251863298, 0.19020419858107102, 91.94826137446736, 71.42412995491115, 29.966563190468133]\n",
      "\n",
      "Testing:  {'learning_rate': 0.03459597445210209, 'max_depth': 8, 'subsample': 0.9825126259140228, 'colsample_bytree': 0.3509339048455815, 'reg_lambda': 56.082313459499446, 'reg_alpha': 6.378669346122511, 'min_child_weight': 17.98902677633556}\n",
      "CV Fold 1 rmse:0.72229 - 910 rounds - it took 32 secs\n",
      "Early stopping for under-performing fold: threshold is 0.72220\n",
      "Last eval:  [0.03459597445210209, 8, 0.9825126259140228, 0.3509339048455815, 56.082313459499446, 6.378669346122511, 17.98902677633556]  - Score  0.7222862137303072\n",
      "Current iter:  32  - Best Score  0.7204004344008961  - Best Args:  [0.26773751831153064, 6, 0.7334997251863298, 0.19020419858107102, 91.94826137446736, 71.42412995491115, 29.966563190468133]\n",
      "\n",
      "Testing:  {'learning_rate': 1.0, 'max_depth': 1, 'subsample': 1.0, 'colsample_bytree': 0.1, 'reg_lambda': 56.316446763428885, 'reg_alpha': 0.0, 'min_child_weight': 30.0}\n",
      "CV Fold 1 rmse:0.72293 - 1780 rounds - it took 34 secs\n",
      "Early stopping for under-performing fold: threshold is 0.72220\n",
      "Last eval:  [1.0, 1, 1.0, 0.1, 56.316446763428885, 0.0, 30.0]  - Score  0.7229270565983851\n",
      "Current iter:  33  - Best Score  0.7204004344008961  - Best Args:  [0.26773751831153064, 6, 0.7334997251863298, 0.19020419858107102, 91.94826137446736, 71.42412995491115, 29.966563190468133]\n",
      "\n",
      "Testing:  {'learning_rate': 0.6043760040818434, 'max_depth': 1, 'subsample': 0.9995377670529898, 'colsample_bytree': 0.13532720235569462, 'reg_lambda': 39.36448708610186, 'reg_alpha': 1.5085613373877487, 'min_child_weight': 29.582782394669405}\n",
      "CV Fold 1 rmse:0.72152 - 5436 rounds - it took 101 secs\n",
      "CV Fold 2 rmse:0.71861 - 5474 rounds - it took 104 secs\n",
      "CV Fold 3 rmse:0.72550 - 2483 rounds - it took 45 secs\n",
      "Early stopping for under-performing fold: threshold is 0.72458\n",
      "Last eval:  [0.6043760040818434, 1, 0.9995377670529898, 0.13532720235569462, 39.36448708610186, 1.5085613373877487, 29.582782394669405]  - Score  0.7218763372807206\n",
      "Current iter:  34  - Best Score  0.7204004344008961  - Best Args:  [0.26773751831153064, 6, 0.7334997251863298, 0.19020419858107102, 91.94826137446736, 71.42412995491115, 29.966563190468133]\n",
      "\n",
      "Testing:  {'learning_rate': 0.4342011971169984, 'max_depth': 1, 'subsample': 1.0, 'colsample_bytree': 0.1, 'reg_lambda': 90.55614113965247, 'reg_alpha': 61.81615983900771, 'min_child_weight': 30.0}\n",
      "CV Fold 1 rmse:0.72338 - 7704 rounds - it took 136 secs\n",
      "Early stopping for under-performing fold: threshold is 0.72204\n",
      "Last eval:  [0.4342011971169984, 1, 1.0, 0.1, 90.55614113965247, 61.81615983900771, 30.0]  - Score  0.7233789102911085\n",
      "Current iter:  35  - Best Score  0.7204004344008961  - Best Args:  [0.26773751831153064, 6, 0.7334997251863298, 0.19020419858107102, 91.94826137446736, 71.42412995491115, 29.966563190468133]\n",
      "\n",
      "Testing:  {'learning_rate': 1.0, 'max_depth': 1, 'subsample': 0.49135553610252336, 'colsample_bytree': 1.0, 'reg_lambda': 91.4697350592225, 'reg_alpha': 100.0, 'min_child_weight': 30.0}\n",
      "CV Fold 1 rmse:0.72272 - 1472 rounds - it took 29 secs\n",
      "Early stopping for under-performing fold: threshold is 0.72204\n",
      "Last eval:  [1.0, 1, 0.49135553610252336, 1.0, 91.4697350592225, 100.0, 30.0]  - Score  0.7227166774941528\n",
      "Current iter:  36  - Best Score  0.7204004344008961  - Best Args:  [0.26773751831153064, 6, 0.7334997251863298, 0.19020419858107102, 91.94826137446736, 71.42412995491115, 29.966563190468133]\n",
      "\n",
      "Testing:  {'learning_rate': 1.0, 'max_depth': 6, 'subsample': 1.0, 'colsample_bytree': 0.1, 'reg_lambda': 83.64766837011216, 'reg_alpha': 100.0, 'min_child_weight': 30.0}\n",
      "CV Fold 1 rmse:0.72242 - 215 rounds - it took 7 secs\n",
      "Early stopping for under-performing fold: threshold is 0.72204\n",
      "Last eval:  [1.0, 6, 1.0, 0.1, 83.64766837011216, 100.0, 30.0]  - Score  0.7224178768710944\n",
      "Current iter:  37  - Best Score  0.7204004344008961  - Best Args:  [0.26773751831153064, 6, 0.7334997251863298, 0.19020419858107102, 91.94826137446736, 71.42412995491115, 29.966563190468133]\n",
      "\n",
      "Testing:  {'learning_rate': 0.03353622302289074, 'max_depth': 7, 'subsample': 0.42294207789648863, 'colsample_bytree': 0.13717044713269477, 'reg_lambda': 98.51922668089067, 'reg_alpha': 4.477111666280465, 'min_child_weight': 9.088918128801168}\n",
      "CV Fold 1 rmse:0.72094 - 1222 rounds - it took 39 secs\n",
      "CV Fold 2 rmse:0.71819 - 1147 rounds - it took 38 secs\n",
      "CV Fold 3 rmse:0.72314 - 1197 rounds - it took 39 secs\n",
      "CV Fold 4 rmse:0.71637 - 1263 rounds - it took 41 secs\n",
      "CV Fold 5 rmse:0.71727 - 1147 rounds - it took 38 secs\n",
      "Last eval:  [0.03353622302289074, 7, 0.42294207789648863, 0.13717044713269477, 98.51922668089067, 4.477111666280465, 9.088918128801168]  - Score  0.7191817376457604\n",
      "Current iter:  38  - Best Score  0.7191817376457604  - Best Args:  [0.03353622302289074, 7, 0.42294207789648863, 0.13717044713269477, 98.51922668089067, 4.477111666280465, 9.088918128801168]\n",
      "\n",
      "Testing:  {'learning_rate': 0.9457236401448835, 'max_depth': 1, 'subsample': 0.8434580552960897, 'colsample_bytree': 0.9823015256864618, 'reg_lambda': 62.08716599814357, 'reg_alpha': 2.5189135062425048, 'min_child_weight': 29.835306462169985}\n",
      "CV Fold 1 rmse:0.72258 - 1049 rounds - it took 20 secs\n",
      "Early stopping for under-performing fold: threshold is 0.72186\n",
      "Last eval:  [0.9457236401448835, 1, 0.8434580552960897, 0.9823015256864618, 62.08716599814357, 2.5189135062425048, 29.835306462169985]  - Score  0.7225799839706714\n",
      "Current iter:  39  - Best Score  0.7191817376457604  - Best Args:  [0.03353622302289074, 7, 0.42294207789648863, 0.13717044713269477, 98.51922668089067, 4.477111666280465, 9.088918128801168]\n",
      "\n",
      "Testing:  {'learning_rate': 0.5949048833117558, 'max_depth': 8, 'subsample': 0.9723357643983517, 'colsample_bytree': 0.10758406835343944, 'reg_lambda': 3.7109237828183104, 'reg_alpha': 1.56413026033615, 'min_child_weight': 9.7538017008124}\n",
      "CV Fold 1 rmse:0.72773 - 28 rounds - it took 4 secs\n",
      "Early stopping for under-performing fold: threshold is 0.72186\n",
      "Last eval:  [0.5949048833117558, 8, 0.9723357643983517, 0.10758406835343944, 3.7109237828183104, 1.56413026033615, 9.7538017008124]  - Score  0.7277269218160334\n",
      "Current iter:  40  - Best Score  0.7191817376457604  - Best Args:  [0.03353622302289074, 7, 0.42294207789648863, 0.13717044713269477, 98.51922668089067, 4.477111666280465, 9.088918128801168]\n",
      "\n",
      "Testing:  {'learning_rate': 0.01, 'max_depth': 1, 'subsample': 0.1, 'colsample_bytree': 0.6195750515839604, 'reg_lambda': 77.82607599511661, 'reg_alpha': 0.0, 'min_child_weight': 30.0}\n",
      "CV Fold 1 rmse:0.72920 - 9509 rounds - it took 170 secs\n",
      "Early stopping for under-performing fold: threshold is 0.72186\n",
      "Last eval:  [0.01, 1, 0.1, 0.6195750515839604, 77.82607599511661, 0.0, 30.0]  - Score  0.7291981529495771\n",
      "Current iter:  41  - Best Score  0.7191817376457604  - Best Args:  [0.03353622302289074, 7, 0.42294207789648863, 0.13717044713269477, 98.51922668089067, 4.477111666280465, 9.088918128801168]\n",
      "\n",
      "Testing:  {'learning_rate': 0.013277852346486721, 'max_depth': 8, 'subsample': 0.49375012700612886, 'colsample_bytree': 0.9588249664082801, 'reg_lambda': 32.002625428030626, 'reg_alpha': 1.798745293866787, 'min_child_weight': 17.518394444757487}\n",
      "CV Fold 1 rmse:0.72394 - 1767 rounds - it took 71 secs\n",
      "Early stopping for under-performing fold: threshold is 0.72186\n",
      "Last eval:  [0.013277852346486721, 8, 0.49375012700612886, 0.9588249664082801, 32.002625428030626, 1.798745293866787, 17.518394444757487]  - Score  0.7239381789744553\n",
      "Current iter:  42  - Best Score  0.7191817376457604  - Best Args:  [0.03353622302289074, 7, 0.42294207789648863, 0.13717044713269477, 98.51922668089067, 4.477111666280465, 9.088918128801168]\n",
      "\n",
      "Testing:  {'learning_rate': 0.023259636797448618, 'max_depth': 7, 'subsample': 0.7361517375756749, 'colsample_bytree': 0.10442918193817347, 'reg_lambda': 8.401268599084611, 'reg_alpha': 99.76351780503245, 'min_child_weight': 4.470283237115474}\n",
      "CV Fold 1 rmse:0.72104 - 8819 rounds - it took 234 secs\n",
      "CV Fold 2 rmse:0.71833 - 7042 rounds - it took 191 secs\n",
      "CV Fold 3 rmse:0.72287 - 8106 rounds - it took 219 secs\n",
      "CV Fold 4 rmse:0.71685 - 6529 rounds - it took 175 secs\n",
      "CV Fold 5 rmse:0.71754 - 8497 rounds - it took 224 secs\n",
      "Last eval:  [0.023259636797448618, 7, 0.7361517375756749, 0.10442918193817347, 8.401268599084611, 99.76351780503245, 4.470283237115474]  - Score  0.7193254162042246\n",
      "Current iter:  43  - Best Score  0.7191817376457604  - Best Args:  [0.03353622302289074, 7, 0.42294207789648863, 0.13717044713269477, 98.51922668089067, 4.477111666280465, 9.088918128801168]\n",
      "\n",
      "Testing:  {'learning_rate': 0.027390549462043577, 'max_depth': 4, 'subsample': 0.5461915881834719, 'colsample_bytree': 0.12302866268465415, 'reg_lambda': 95.77042280316803, 'reg_alpha': 42.27276737034686, 'min_child_weight': 28.2243819355067}\n",
      "CV Fold 1 rmse:0.72059 - 5295 rounds - it took 126 secs\n",
      "CV Fold 2 rmse:0.71767 - 5582 rounds - it took 135 secs\n",
      "CV Fold 3 rmse:0.72250 - 5181 rounds - it took 127 secs\n",
      "CV Fold 4 rmse:0.71577 - 6254 rounds - it took 159 secs\n",
      "CV Fold 5 rmse:0.71727 - 4570 rounds - it took 113 secs\n",
      "Last eval:  [0.027390549462043577, 4, 0.5461915881834719, 0.12302866268465415, 95.77042280316803, 42.27276737034686, 28.2243819355067]  - Score  0.7187592465142455\n",
      "Current iter:  44  - Best Score  0.7187592465142455  - Best Args:  [0.027390549462043577, 4, 0.5461915881834719, 0.12302866268465415, 95.77042280316803, 42.27276737034686, 28.2243819355067]\n",
      "\n",
      "Testing:  {'learning_rate': 0.9739389117798737, 'max_depth': 4, 'subsample': 0.9898683623568275, 'colsample_bytree': 0.9127209520559293, 'reg_lambda': 15.805646874605786, 'reg_alpha': 52.11258477962466, 'min_child_weight': 19.866076664121568}\n",
      "CV Fold 1 rmse:0.72707 - 92 rounds - it took 5 secs\n",
      "Early stopping for under-performing fold: threshold is 0.72165\n",
      "Last eval:  [0.9739389117798737, 4, 0.9898683623568275, 0.9127209520559293, 15.805646874605786, 52.11258477962466, 19.866076664121568]  - Score  0.7270730450818624\n",
      "Current iter:  45  - Best Score  0.7187592465142455  - Best Args:  [0.027390549462043577, 4, 0.5461915881834719, 0.12302866268465415, 95.77042280316803, 42.27276737034686, 28.2243819355067]\n",
      "\n",
      "Testing:  {'learning_rate': 0.02082684995666582, 'max_depth': 4, 'subsample': 0.5570601378688809, 'colsample_bytree': 0.8502206504815417, 'reg_lambda': 28.88271491805135, 'reg_alpha': 90.72373291545176, 'min_child_weight': 2.1517718375886625}\n",
      "CV Fold 1 rmse:0.72296 - 5207 rounds - it took 140 secs\n",
      "Early stopping for under-performing fold: threshold is 0.72165\n",
      "Last eval:  [0.02082684995666582, 4, 0.5570601378688809, 0.8502206504815417, 28.88271491805135, 90.72373291545176, 2.1517718375886625]  - Score  0.7229637091942341\n",
      "Current iter:  46  - Best Score  0.7187592465142455  - Best Args:  [0.027390549462043577, 4, 0.5461915881834719, 0.12302866268465415, 95.77042280316803, 42.27276737034686, 28.2243819355067]\n",
      "\n",
      "Testing:  {'learning_rate': 0.05510311009169886, 'max_depth': 5, 'subsample': 0.8014468359263781, 'colsample_bytree': 0.1421072101038539, 'reg_lambda': 12.25861183647913, 'reg_alpha': 2.683748226012107, 'min_child_weight': 14.932922484857816}\n",
      "CV Fold 1 rmse:0.72075 - 1151 rounds - it took 32 secs\n",
      "CV Fold 2 rmse:0.71817 - 1168 rounds - it took 31 secs\n",
      "CV Fold 3 rmse:0.72261 - 1194 rounds - it took 32 secs\n",
      "CV Fold 4 rmse:0.71639 - 1170 rounds - it took 32 secs\n",
      "CV Fold 5 rmse:0.71693 - 1164 rounds - it took 32 secs\n",
      "Last eval:  [0.05510311009169886, 5, 0.8014468359263781, 0.1421072101038539, 12.25861183647913, 2.683748226012107, 14.932922484857816]  - Score  0.7189720618294506\n",
      "Current iter:  47  - Best Score  0.7187592465142455  - Best Args:  [0.027390549462043577, 4, 0.5461915881834719, 0.12302866268465415, 95.77042280316803, 42.27276737034686, 28.2243819355067]\n",
      "\n",
      "Testing:  {'learning_rate': 0.01, 'max_depth': 6, 'subsample': 0.6260870682455776, 'colsample_bytree': 0.1, 'reg_lambda': 12.245679527079673, 'reg_alpha': 16.64351161342476, 'min_child_weight': 30.0}\n",
      "CV Fold 1 rmse:0.72011 - 6336 rounds - it took 181 secs\n",
      "CV Fold 2 rmse:0.71733 - 6319 rounds - it took 176 secs\n",
      "CV Fold 3 rmse:0.72213 - 6072 rounds - it took 170 secs\n",
      "CV Fold 4 rmse:0.71551 - 6534 rounds - it took 180 secs\n",
      "CV Fold 5 rmse:0.71635 - 6585 rounds - it took 183 secs\n",
      "Last eval:  [0.01, 6, 0.6260870682455776, 0.1, 12.245679527079673, 16.64351161342476, 30.0]  - Score  0.7182874710305741\n",
      "Current iter:  48  - Best Score  0.7182874710305741  - Best Args:  [0.01, 6, 0.6260870682455776, 0.1, 12.245679527079673, 16.64351161342476, 30.0]\n",
      "\n",
      "Testing:  {'learning_rate': 0.0164294573812949, 'max_depth': 8, 'subsample': 0.9855867750961604, 'colsample_bytree': 0.7804613942950032, 'reg_lambda': 83.23128090711054, 'reg_alpha': 96.09200750104614, 'min_child_weight': 25.400239503908264}\n",
      "CV Fold 1 rmse:0.72307 - 3020 rounds - it took 109 secs\n",
      "Early stopping for under-performing fold: threshold is 0.72140\n",
      "Last eval:  [0.0164294573812949, 8, 0.9855867750961604, 0.7804613942950032, 83.23128090711054, 96.09200750104614, 25.400239503908264]  - Score  0.7230737033385001\n",
      "Current iter:  49  - Best Score  0.7182874710305741  - Best Args:  [0.01, 6, 0.6260870682455776, 0.1, 12.245679527079673, 16.64351161342476, 30.0]\n",
      "\n",
      "Testing:  {'learning_rate': 0.01, 'max_depth': 6, 'subsample': 0.47315601250829264, 'colsample_bytree': 0.1, 'reg_lambda': 100.0, 'reg_alpha': 100.0, 'min_child_weight': 1.1247149452092275}\n",
      "CV Fold 1 rmse:0.72291 - 9998 rounds - it took 262 secs\n",
      "Early stopping for under-performing fold: threshold is 0.72140\n",
      "Last eval:  [0.01, 6, 0.47315601250829264, 0.1, 100.0, 100.0, 1.1247149452092275]  - Score  0.7229124523010838\n",
      "Current iter:  50  - Best Score  0.7182874710305741  - Best Args:  [0.01, 6, 0.6260870682455776, 0.1, 12.245679527079673, 16.64351161342476, 30.0]\n",
      "\n",
      "Testing:  {'learning_rate': 0.40139346935615866, 'max_depth': 4, 'subsample': 0.9628685963368061, 'colsample_bytree': 0.13523857738740225, 'reg_lambda': 40.038821270654445, 'reg_alpha': 99.89447265433432, 'min_child_weight': 11.089923355080918}\n",
      "CV Fold 1 rmse:0.72179 - 513 rounds - it took 15 secs\n",
      "Early stopping for under-performing fold: threshold is 0.72140\n",
      "Last eval:  [0.40139346935615866, 4, 0.9628685963368061, 0.13523857738740225, 40.038821270654445, 99.89447265433432, 11.089923355080918]  - Score  0.7217889530030179\n",
      "Current iter:  51  - Best Score  0.7182874710305741  - Best Args:  [0.01, 6, 0.6260870682455776, 0.1, 12.245679527079673, 16.64351161342476, 30.0]\n",
      "\n",
      "Testing:  {'learning_rate': 0.8507441131416335, 'max_depth': 1, 'subsample': 0.1272228397009001, 'colsample_bytree': 0.9645739697051062, 'reg_lambda': 67.32755737689268, 'reg_alpha': 3.2319957246445217, 'min_child_weight': 8.518374983284925}\n",
      "CV Fold 1 rmse:0.72678 - 495 rounds - it took 10 secs\n",
      "Early stopping for under-performing fold: threshold is 0.72140\n",
      "Last eval:  [0.8507441131416335, 1, 0.1272228397009001, 0.9645739697051062, 67.32755737689268, 3.2319957246445217, 8.518374983284925]  - Score  0.72678098132575\n",
      "Current iter:  52  - Best Score  0.7182874710305741  - Best Args:  [0.01, 6, 0.6260870682455776, 0.1, 12.245679527079673, 16.64351161342476, 30.0]\n",
      "\n",
      "Testing:  {'learning_rate': 0.01, 'max_depth': 4, 'subsample': 0.5009186557234353, 'colsample_bytree': 0.1, 'reg_lambda': 16.677186672570514, 'reg_alpha': 0.0, 'min_child_weight': 12.500186459175387}\n",
      "CV Fold 1 rmse:0.72055 - 9547 rounds - it took 222 secs\n",
      "CV Fold 2 rmse:0.71783 - 8341 rounds - it took 194 secs\n",
      "CV Fold 3 rmse:0.72262 - 8114 rounds - it took 194 secs\n",
      "CV Fold 4 rmse:0.71609 - 8634 rounds - it took 205 secs\n",
      "CV Fold 5 rmse:0.71651 - 9559 rounds - it took 224 secs\n",
      "Last eval:  [0.01, 4, 0.5009186557234353, 0.1, 16.677186672570514, 0.0, 12.500186459175387]  - Score  0.7187198461786737\n",
      "Current iter:  53  - Best Score  0.7182874710305741  - Best Args:  [0.01, 6, 0.6260870682455776, 0.1, 12.245679527079673, 16.64351161342476, 30.0]\n",
      "\n",
      "Testing:  {'learning_rate': 0.9978245074753869, 'max_depth': 4, 'subsample': 0.4902294044183533, 'colsample_bytree': 0.25308324619330874, 'reg_lambda': 17.38058767439608, 'reg_alpha': 87.09490286664503, 'min_child_weight': 17.4268129515866}\n",
      "CV Fold 1 rmse:0.72671 - 110 rounds - it took 4 secs\n",
      "Early stopping for under-performing fold: threshold is 0.72104\n",
      "Last eval:  [0.9978245074753869, 4, 0.4902294044183533, 0.25308324619330874, 17.38058767439608, 87.09490286664503, 17.4268129515866]  - Score  0.7267116699975317\n",
      "Current iter:  54  - Best Score  0.7182874710305741  - Best Args:  [0.01, 6, 0.6260870682455776, 0.1, 12.245679527079673, 16.64351161342476, 30.0]\n",
      "\n",
      "Testing:  {'learning_rate': 0.017956504074879573, 'max_depth': 5, 'subsample': 0.2408808616800931, 'colsample_bytree': 0.8804711222107261, 'reg_lambda': 50.48508421515103, 'reg_alpha': 3.2336948499009597, 'min_child_weight': 24.130218840856514}\n",
      "CV Fold 1 rmse:0.72347 - 2124 rounds - it took 59 secs\n",
      "Early stopping for under-performing fold: threshold is 0.72104\n",
      "Last eval:  [0.017956504074879573, 5, 0.2408808616800931, 0.8804711222107261, 50.48508421515103, 3.2336948499009597, 24.130218840856514]  - Score  0.7234661488371932\n",
      "Current iter:  55  - Best Score  0.7182874710305741  - Best Args:  [0.01, 6, 0.6260870682455776, 0.1, 12.245679527079673, 16.64351161342476, 30.0]\n",
      "\n",
      "Testing:  {'learning_rate': 0.01, 'max_depth': 6, 'subsample': 1.0, 'colsample_bytree': 0.1, 'reg_lambda': 97.14130929166086, 'reg_alpha': 100.0, 'min_child_weight': 21.80651802018074}\n",
      "CV Fold 1 rmse:0.72151 - 9999 rounds - it took 255 secs\n",
      "Early stopping for under-performing fold: threshold is 0.72104\n",
      "Last eval:  [0.01, 6, 1.0, 0.1, 97.14130929166086, 100.0, 21.80651802018074]  - Score  0.7215094612709092\n",
      "Current iter:  56  - Best Score  0.7182874710305741  - Best Args:  [0.01, 6, 0.6260870682455776, 0.1, 12.245679527079673, 16.64351161342476, 30.0]\n",
      "\n",
      "Testing:  {'learning_rate': 0.9815313394556654, 'max_depth': 8, 'subsample': 0.9939376292712923, 'colsample_bytree': 0.8968756342939637, 'reg_lambda': 87.51134290767546, 'reg_alpha': 19.747954220325663, 'min_child_weight': 14.883144307174636}\n",
      "CV Fold 1 rmse:0.73567 - 8 rounds - it took 3 secs\n",
      "Early stopping for under-performing fold: threshold is 0.72104\n",
      "Last eval:  [0.9815313394556654, 8, 0.9939376292712923, 0.8968756342939637, 87.51134290767546, 19.747954220325663, 14.883144307174636]  - Score  0.7356689390559624\n",
      "Current iter:  57  - Best Score  0.7182874710305741  - Best Args:  [0.01, 6, 0.6260870682455776, 0.1, 12.245679527079673, 16.64351161342476, 30.0]\n",
      "\n",
      "Testing:  {'learning_rate': 0.4371684402652018, 'max_depth': 8, 'subsample': 0.9919590610383998, 'colsample_bytree': 0.20388674490987443, 'reg_lambda': 80.52568487824806, 'reg_alpha': 96.95138789313606, 'min_child_weight': 14.466437591886356}\n",
      "CV Fold 1 rmse:0.72298 - 226 rounds - it took 10 secs\n",
      "Early stopping for under-performing fold: threshold is 0.72104\n",
      "Last eval:  [0.4371684402652018, 8, 0.9919590610383998, 0.20388674490987443, 80.52568487824806, 96.95138789313606, 14.466437591886356]  - Score  0.7229826727179095\n",
      "Current iter:  58  - Best Score  0.7182874710305741  - Best Args:  [0.01, 6, 0.6260870682455776, 0.1, 12.245679527079673, 16.64351161342476, 30.0]\n",
      "\n",
      "Testing:  {'learning_rate': 0.028142000687413883, 'max_depth': 1, 'subsample': 0.7088508165304099, 'colsample_bytree': 0.8773221684169189, 'reg_lambda': 98.61397750986764, 'reg_alpha': 1.205453154508263, 'min_child_weight': 29.0872809110154}\n",
      "CV Fold 1 rmse:0.72687 - 9999 rounds - it took 181 secs\n",
      "Early stopping for under-performing fold: threshold is 0.72104\n",
      "Last eval:  [0.028142000687413883, 1, 0.7088508165304099, 0.8773221684169189, 98.61397750986764, 1.205453154508263, 29.0872809110154]  - Score  0.7268670017017997\n",
      "Current iter:  59  - Best Score  0.7182874710305741  - Best Args:  [0.01, 6, 0.6260870682455776, 0.1, 12.245679527079673, 16.64351161342476, 30.0]\n",
      "\n",
      "Testing:  {'learning_rate': 0.9641424440840559, 'max_depth': 1, 'subsample': 0.9908417088722442, 'colsample_bytree': 0.9800192735013677, 'reg_lambda': 20.433156667211122, 'reg_alpha': 87.37134822551226, 'min_child_weight': 11.647661078493634}\n",
      "CV Fold 1 rmse:0.72233 - 5237 rounds - it took 94 secs\n",
      "Early stopping for under-performing fold: threshold is 0.72104\n",
      "Last eval:  [0.9641424440840559, 1, 0.9908417088722442, 0.9800192735013677, 20.433156667211122, 87.37134822551226, 11.647661078493634]  - Score  0.7223294779022024\n",
      "Current iter:  60  - Best Score  0.7182874710305741  - Best Args:  [0.01, 6, 0.6260870682455776, 0.1, 12.245679527079673, 16.64351161342476, 30.0]\n"
     ]
    }
   ],
   "source": [
    "x0, y0 = joblib.load('checkpoint.pkl')\n",
    "\n",
    "forest_round = forest_minimize(func=objective,\n",
    "                               x0=x0, # already examined values for x\n",
    "                               y0=y0, # observed values for x0\n",
    "                               dimensions=space,\n",
    "                               n_calls=30,\n",
    "                               n_initial_points=0,\n",
    "                               callback=[onstep],\n",
    "                               random_state=0)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "id": "7395da70",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-10-05T20:14:59.248423Z",
     "iopub.status.busy": "2024-10-05T20:14:59.247970Z",
     "iopub.status.idle": "2024-10-05T20:14:59.256725Z",
     "shell.execute_reply": "2024-10-05T20:14:59.255715Z"
    },
    "papermill": {
     "duration": 0.027734,
     "end_time": "2024-10-05T20:14:59.259036",
     "exception": false,
     "start_time": "2024-10-05T20:14:59.231302",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "60\n"
     ]
    }
   ],
   "source": [
    "x0, y0 = joblib.load('checkpoint.pkl')\n",
    "print(len(x0))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "id": "ed5ed991",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-10-05T20:14:59.292858Z",
     "iopub.status.busy": "2024-10-05T20:14:59.291721Z",
     "iopub.status.idle": "2024-10-05T20:14:59.298208Z",
     "shell.execute_reply": "2024-10-05T20:14:59.297068Z"
    },
    "papermill": {
     "duration": 0.026774,
     "end_time": "2024-10-05T20:14:59.300880",
     "exception": false,
     "start_time": "2024-10-05T20:14:59.274106",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Best score: 0.71829\n",
      "Best hyperparameters:\n",
      "learning_rate             : 0.01\n",
      "max_depth                 : 6\n",
      "subsample                 : 0.6260870682455776\n",
      "colsample_bytree          : 0.1\n",
      "reg_lambda                : 12.245679527079673\n",
      "reg_alpha                 : 16.64351161342476\n",
      "min_child_weight          : 30.0\n"
     ]
    }
   ],
   "source": [
    "print(f\"Best score: {forest_round.fun:0.5f}\")\n",
    "print(\"Best hyperparameters:\")\n",
    "for sp, x in zip(forest_round.space, forest_round.x):\n",
    "    print(f\"{sp.name:25} : {x}\")"
   ]
  }
 ],
 "metadata": {
  "kaggle": {
   "accelerator": "none",
   "dataSources": [
    {
     "databundleVersionId": 2267921,
     "sourceId": 27423,
     "sourceType": "competition"
    }
   ],
   "isGpuEnabled": false,
   "isInternetEnabled": true,
   "language": "python",
   "sourceType": "notebook"
  },
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.10.14"
  },
  "papermill": {
   "default_parameters": {},
   "duration": 7966.632227,
   "end_time": "2024-10-05T20:15:00.041226",
   "environment_variables": {},
   "exception": null,
   "input_path": "__notebook__.ipynb",
   "output_path": "__notebook__.ipynb",
   "parameters": {},
   "start_time": "2024-10-05T18:02:13.408999",
   "version": "2.6.0"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}
